{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程\n",
    "任务简介： \n",
    "广告点击率（Click-Through Rate Prediction, CTR）是互联网计算广告中的关键环节，预估准确性直接影响公司广告收入。机器学习技术可在计算广告中大展身手，Avazu通过程序化广告技术进行效果营销。本项目我们对Avazu提供的Kaggle竞赛数据进行移动CTR预估，其Kaggle竞赛网页为：https://www.kaggle.com/c/avazu-ctr-prediction。 \n",
    "\n",
    "属性：\n",
    "id: ad identifier （ID）\n",
    "click: 0/1 for non-click/click (是否被点击 0否、1是)\n",
    "hour: format is YYMMDDHH, so 14091123 means 23:00 on Sept. 11, 2014 UTC.（时间）\n",
    "C1 -- anonymized categorical variable（类型变量）\n",
    "banner_pos （广告位置）\n",
    "site_id   （站点ID）\n",
    "site_domain (站点领域)\n",
    "site_category  （站点类别）\n",
    "app_id   （APP_ID）\n",
    "app_domain (APP_领域)\n",
    "app_category （APP_类别）\n",
    "device_id (设备ID)\n",
    "device_ip (设备IP)\n",
    "device_model （设备模型）\n",
    "device_type （设备类型）\n",
    "device_conn_type （设备连接类型）\n",
    "C14-C21 -- anonymized categorical variables （（类型变量））"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#导包\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#读取数据\n",
    "train = pd.read_csv(\"train.csv\",nrows=1000000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 24)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.sample(100000) #随机抽取10w数据\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>click</th>\n",
       "      <th>hour</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>site_id</th>\n",
       "      <th>site_domain</th>\n",
       "      <th>site_category</th>\n",
       "      <th>app_id</th>\n",
       "      <th>app_domain</th>\n",
       "      <th>...</th>\n",
       "      <th>device_type</th>\n",
       "      <th>device_conn_type</th>\n",
       "      <th>C14</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>186848</th>\n",
       "      <td>1.819859e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>14102101</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>85f751fd</td>\n",
       "      <td>c4e18dd6</td>\n",
       "      <td>50e219e0</td>\n",
       "      <td>9c13b419</td>\n",
       "      <td>2347f47a</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>20633</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2374</td>\n",
       "      <td>3</td>\n",
       "      <td>39</td>\n",
       "      <td>-1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815809</th>\n",
       "      <td>3.357563e+18</td>\n",
       "      <td>0</td>\n",
       "      <td>14102104</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15706</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>-1</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51654</th>\n",
       "      <td>1.719468e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>14102100</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>85f751fd</td>\n",
       "      <td>c4e18dd6</td>\n",
       "      <td>50e219e0</td>\n",
       "      <td>febd1138</td>\n",
       "      <td>82e27996</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21611</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2480</td>\n",
       "      <td>3</td>\n",
       "      <td>297</td>\n",
       "      <td>100111</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979514</th>\n",
       "      <td>1.349057e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>14102105</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>85f751fd</td>\n",
       "      <td>c4e18dd6</td>\n",
       "      <td>50e219e0</td>\n",
       "      <td>9c13b419</td>\n",
       "      <td>2347f47a</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19251</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2201</td>\n",
       "      <td>3</td>\n",
       "      <td>35</td>\n",
       "      <td>-1</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58702</th>\n",
       "      <td>1.817488e+19</td>\n",
       "      <td>1</td>\n",
       "      <td>14102100</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>d9750ee7</td>\n",
       "      <td>98572c79</td>\n",
       "      <td>f028772b</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15704</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>-1</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id  click      hour    C1  banner_pos   site_id site_domain  \\\n",
       "186848  1.819859e+19      0  14102101  1005           0  85f751fd    c4e18dd6   \n",
       "815809  3.357563e+18      0  14102104  1005           0  1fbe01fe    f3845767   \n",
       "51654   1.719468e+19      0  14102100  1005           0  85f751fd    c4e18dd6   \n",
       "979514  1.349057e+19      0  14102105  1005           0  85f751fd    c4e18dd6   \n",
       "58702   1.817488e+19      1  14102100  1005           0  d9750ee7    98572c79   \n",
       "\n",
       "       site_category    app_id app_domain ...  device_type device_conn_type  \\\n",
       "186848      50e219e0  9c13b419   2347f47a ...            1                0   \n",
       "815809      28905ebd  ecad2386   7801e8d9 ...            1                0   \n",
       "51654       50e219e0  febd1138   82e27996 ...            1                0   \n",
       "979514      50e219e0  9c13b419   2347f47a ...            1                0   \n",
       "58702       f028772b  ecad2386   7801e8d9 ...            1                0   \n",
       "\n",
       "          C14  C15  C16   C17  C18  C19     C20  C21  \n",
       "186848  20633  320   50  2374    3   39      -1   23  \n",
       "815809  15706  320   50  1722    0   35      -1   79  \n",
       "51654   21611  320   50  2480    3  297  100111   61  \n",
       "979514  19251  320   50  2201    3   35      -1   43  \n",
       "58702   15704  320   50  1722    0   35      -1   79  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    83852\n",
       "1    16148\n",
       "Name: click, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看正负样本比例(已经经过了下采样)\n",
    "lable = train['click'].value_counts()\n",
    "lable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 23)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取验证样本\n",
    "test = pd.read_csv('test',nrows=1000000)\n",
    "test = test.sample(100000)\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hour</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>site_id</th>\n",
       "      <th>site_domain</th>\n",
       "      <th>site_category</th>\n",
       "      <th>app_id</th>\n",
       "      <th>app_domain</th>\n",
       "      <th>app_category</th>\n",
       "      <th>...</th>\n",
       "      <th>device_type</th>\n",
       "      <th>device_conn_type</th>\n",
       "      <th>C14</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>536962</th>\n",
       "      <td>1.060423e+18</td>\n",
       "      <td>14103105</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>4e7614cf</td>\n",
       "      <td>c1aa3c04</td>\n",
       "      <td>f028772b</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>07d7df22</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8330</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>761</td>\n",
       "      <td>3</td>\n",
       "      <td>175</td>\n",
       "      <td>100075</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736381</th>\n",
       "      <td>1.368686e+17</td>\n",
       "      <td>14103106</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>07d7df22</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>24084</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2761</td>\n",
       "      <td>2</td>\n",
       "      <td>35</td>\n",
       "      <td>100148</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330635</th>\n",
       "      <td>1.457154e+19</td>\n",
       "      <td>14103103</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>07d7df22</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>23137</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2664</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>-1</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858388</th>\n",
       "      <td>4.625780e+18</td>\n",
       "      <td>14103106</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1fbe01fe</td>\n",
       "      <td>f3845767</td>\n",
       "      <td>28905ebd</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>07d7df22</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>22254</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2545</td>\n",
       "      <td>0</td>\n",
       "      <td>431</td>\n",
       "      <td>-1</td>\n",
       "      <td>221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>786323</th>\n",
       "      <td>1.677718e+19</td>\n",
       "      <td>14103106</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>2c5c874d</td>\n",
       "      <td>9d54950b</td>\n",
       "      <td>e787de0e</td>\n",
       "      <td>ecad2386</td>\n",
       "      <td>7801e8d9</td>\n",
       "      <td>07d7df22</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24132</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2768</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>100189</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id      hour    C1  banner_pos   site_id site_domain  \\\n",
       "536962  1.060423e+18  14103105  1005           0  4e7614cf    c1aa3c04   \n",
       "736381  1.368686e+17  14103106  1005           0  1fbe01fe    f3845767   \n",
       "330635  1.457154e+19  14103103  1005           0  1fbe01fe    f3845767   \n",
       "858388  4.625780e+18  14103106  1005           0  1fbe01fe    f3845767   \n",
       "786323  1.677718e+19  14103106  1005           0  2c5c874d    9d54950b   \n",
       "\n",
       "       site_category    app_id app_domain app_category ...  device_type  \\\n",
       "536962      f028772b  ecad2386   7801e8d9     07d7df22 ...            1   \n",
       "736381      28905ebd  ecad2386   7801e8d9     07d7df22 ...            1   \n",
       "330635      28905ebd  ecad2386   7801e8d9     07d7df22 ...            1   \n",
       "858388      28905ebd  ecad2386   7801e8d9     07d7df22 ...            1   \n",
       "786323      e787de0e  ecad2386   7801e8d9     07d7df22 ...            1   \n",
       "\n",
       "       device_conn_type    C14  C15  C16   C17  C18  C19     C20  C21  \n",
       "536962                0   8330  320   50   761    3  175  100075   23  \n",
       "736381                2  24084  320   50  2761    2   35  100148   13  \n",
       "330635                0  23137  320   50  2664    0   35      -1   51  \n",
       "858388                0  22254  320   50  2545    0  431      -1  221  \n",
       "786323                0  24132  320   50  2768    1   33  100189   71  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "site_id属性的不同取值和出现的次数\n",
      "85f751fd    38980\n",
      "1fbe01fe    24898\n",
      "e151e245     3187\n",
      "5b4d2eda     2241\n",
      "5b08c53b     1929\n",
      "d9750ee7     1614\n",
      "1c3a1b95      918\n",
      "2c5c874d      885\n",
      "b7e9786d      701\n",
      "57ef2c87      697\n",
      "d6137915      653\n",
      "856e6d3f      646\n",
      "17caea14      629\n",
      "57fe1b20      624\n",
      "e8f79e60      621\n",
      "5ee41ff2      598\n",
      "83a0ad1a      576\n",
      "6399eda6      546\n",
      "4ba33bb6      534\n",
      "6ec06dbd      521\n",
      "e4d8dd7b      513\n",
      "0a742914      503\n",
      "4e7614cf      483\n",
      "93eaba74      462\n",
      "6256f5b4      400\n",
      "b99a2c43      385\n",
      "e3c09f3a      340\n",
      "0eb72673      281\n",
      "88154ade      276\n",
      "e5c60a05      275\n",
      "            ...  \n",
      "09b74f3a        1\n",
      "f908f7e6        1\n",
      "98902b4d        1\n",
      "c29eaee8        1\n",
      "acc650b1        1\n",
      "33de3cff        1\n",
      "76f70dde        1\n",
      "40165ece        1\n",
      "b8561528        1\n",
      "79d1bbff        1\n",
      "1289c10f        1\n",
      "ee4e7db8        1\n",
      "90a340b2        1\n",
      "3dff2c35        1\n",
      "86611e43        1\n",
      "cb7d700e        1\n",
      "3136689e        1\n",
      "84aff0b1        1\n",
      "30a3a37d        1\n",
      "8dfecaca        1\n",
      "079325ff        1\n",
      "397808ec        1\n",
      "3b91ce35        1\n",
      "451bd647        1\n",
      "b26c34cc        1\n",
      "2ccb7139        1\n",
      "3e447381        1\n",
      "030095a6        1\n",
      "fbefe81c        1\n",
      "1c82a473        1\n",
      "Name: site_id, Length: 1142, dtype: int64\n",
      "\n",
      "site_domain属性的不同取值和出现的次数\n",
      "c4e18dd6    40139\n",
      "f3845767    24898\n",
      "7e091613     3232\n",
      "7687a86e     2665\n",
      "16a36ef3     2380\n",
      "98572c79     1622\n",
      "9d54950b     1056\n",
      "2d235ae0      920\n",
      "d262cf1e      861\n",
      "b12b9f85      702\n",
      "bd6d812f      697\n",
      "bb1ef334      653\n",
      "58a89a43      646\n",
      "0dde25ec      629\n",
      "5b626596      624\n",
      "c4342784      621\n",
      "17d996e6      602\n",
      "5c9ae867      576\n",
      "968765cd      546\n",
      "e2696795      535\n",
      "a17bde68      513\n",
      "510bd839      503\n",
      "c1aa3c04      502\n",
      "28f93029      400\n",
      "cc962a1f      385\n",
      "6b59f079      334\n",
      "7256c623      331\n",
      "9f448720      292\n",
      "d2f72222      281\n",
      "06908927      266\n",
      "            ...  \n",
      "5dfd17c3        1\n",
      "7bc893bf        1\n",
      "4d361975        1\n",
      "71205ef1        1\n",
      "64e0e123        1\n",
      "f4b59897        1\n",
      "3baa6a0d        1\n",
      "98ae9b4f        1\n",
      "ed48e727        1\n",
      "06fd1a96        1\n",
      "db6b5705        1\n",
      "43d9c8f2        1\n",
      "5e19d579        1\n",
      "983b49d4        1\n",
      "bd694c06        1\n",
      "84156eef        1\n",
      "96f8503e        1\n",
      "18270dc4        1\n",
      "a0c97bcc        1\n",
      "45214ca1        1\n",
      "dbdbeadb        1\n",
      "3ae98fc6        1\n",
      "383e39a8        1\n",
      "e81ee181        1\n",
      "455521b7        1\n",
      "e6e82c06        1\n",
      "ef69a0aa        1\n",
      "950201e9        1\n",
      "226288fb        1\n",
      "a82bd0a5        1\n",
      "Name: site_domain, Length: 1046, dtype: int64\n",
      "\n",
      "site_category属性的不同取值和出现的次数\n",
      "50e219e0    42657\n",
      "28905ebd    26103\n",
      "f028772b    21491\n",
      "3e814130     6087\n",
      "e787de0e      885\n",
      "75fa27f6      850\n",
      "335d28a8      577\n",
      "8fd0aea4      523\n",
      "f66779e6      250\n",
      "76b2941d      151\n",
      "c0dd3be3      130\n",
      "dedf689d      114\n",
      "72722551       78\n",
      "a818d37a       34\n",
      "70fb0e29       32\n",
      "0569f928       29\n",
      "42a36e14        7\n",
      "5378d028        2\n",
      "Name: site_category, dtype: int64\n",
      "\n",
      "app_id属性的不同取值和出现的次数\n",
      "ecad2386    61020\n",
      "685d1c4c     6009\n",
      "9c13b419     5795\n",
      "febd1138     2876\n",
      "e2fcccd2     2338\n",
      "3c4b944d     1172\n",
      "92f5800b      942\n",
      "d36838b1      902\n",
      "7e7baafa      840\n",
      "ce183bbd      829\n",
      "cf0327f9      816\n",
      "7358e05e      752\n",
      "98fed791      705\n",
      "e9739828      689\n",
      "51cedd4e      598\n",
      "03528b27      572\n",
      "f0d41ff1      569\n",
      "e2a1ca37      553\n",
      "1dc72b4d      528\n",
      "54c5d545      527\n",
      "f1cd0776      455\n",
      "be7c618d      379\n",
      "53de0284      358\n",
      "396df801      336\n",
      "f888bf4c      276\n",
      "d44c074c      261\n",
      "ffc6ffd0      245\n",
      "a5184c22      215\n",
      "0acbeaa3      199\n",
      "de97da65      167\n",
      "            ...  \n",
      "8d657e99        1\n",
      "c9480079        1\n",
      "d1f14f7d        1\n",
      "4afbcb2f        1\n",
      "5e677b98        1\n",
      "9b016ac2        1\n",
      "3e5ac0cf        1\n",
      "7b562ef1        1\n",
      "9c1a0df5        1\n",
      "d768ebc5        1\n",
      "9db7e3c6        1\n",
      "bd4dc570        1\n",
      "d38d5c3d        1\n",
      "7eca0ef1        1\n",
      "6570d09a        1\n",
      "26f0aceb        1\n",
      "d6598897        1\n",
      "5f5ec5af        1\n",
      "f62d597f        1\n",
      "79450f15        1\n",
      "b16f5d8d        1\n",
      "cd06c6c3        1\n",
      "24258780        1\n",
      "f59a9ed7        1\n",
      "e51dfc9f        1\n",
      "6bc7ebab        1\n",
      "c67fa2ba        1\n",
      "f8f8e0be        1\n",
      "75ae11c1        1\n",
      "6ae09b45        1\n",
      "Name: app_id, Length: 958, dtype: int64\n",
      "\n",
      "app_domain属性的不同取值和出现的次数\n",
      "7801e8d9    63665\n",
      "2347f47a    20971\n",
      "82e27996     2876\n",
      "ae637522     2711\n",
      "d9b5648e     2389\n",
      "5c5a694b     2342\n",
      "0e8616ad      934\n",
      "b9528b13      781\n",
      "df32afa9      689\n",
      "aefc06bd      676\n",
      "33da2e74      413\n",
      "5b9c592b      276\n",
      "b8d325c3      231\n",
      "45a51db4      199\n",
      "b5f3b24a      163\n",
      "6f7ca2ba      137\n",
      "1ea19ec4      101\n",
      "5c620f04       85\n",
      "f2f777fb       45\n",
      "ad63ec9b       38\n",
      "448ca2e3       35\n",
      "15ec7f39       34\n",
      "0654b444       32\n",
      "813f3323       29\n",
      "c6824def       19\n",
      "2b627705       16\n",
      "a8b0bf20       16\n",
      "828da833       15\n",
      "5b3f66ff       12\n",
      "9ec164d3       11\n",
      "db829551        7\n",
      "1ed56ded        5\n",
      "b408d42a        4\n",
      "5ac0b939        4\n",
      "15c23f8e        4\n",
      "d6feb1a4        3\n",
      "fd5f0ee2        3\n",
      "f435cae0        3\n",
      "afdf1f54        3\n",
      "7a9371fa        2\n",
      "5ec95754        2\n",
      "6a0a3a9d        2\n",
      "18eb4e75        2\n",
      "3fa331b0        2\n",
      "bd8c1fdc        1\n",
      "63f57be0        1\n",
      "b7bbc1c1        1\n",
      "519a450d        1\n",
      "2c1c31c6        1\n",
      "b7af3e0a        1\n",
      "cda96d46        1\n",
      "f5a7c834        1\n",
      "b299335a        1\n",
      "d95432fe        1\n",
      "449e219f        1\n",
      "a841febe        1\n",
      "885c7f3f        1\n",
      "Name: app_domain, dtype: int64\n",
      "\n",
      "app_category属性的不同取值和出现的次数\n",
      "07d7df22    61528\n",
      "0f2161f8    19897\n",
      "8ded1f7a     7917\n",
      "f95efa07     6624\n",
      "cef3e649     3537\n",
      "dc97ec06      107\n",
      "d1327cf5      105\n",
      "75d80bbe       60\n",
      "4ce2e9fc       59\n",
      "fc6fa53d       58\n",
      "09481d60       23\n",
      "879c24eb       19\n",
      "0f9a328c       16\n",
      "a86a3e89       13\n",
      "a3c42688       11\n",
      "8df2e842        7\n",
      "2281a340        7\n",
      "4681bb9d        4\n",
      "2fc4f2aa        4\n",
      "79f0b860        2\n",
      "7113d72a        1\n",
      "4b7ade46        1\n",
      "Name: app_category, dtype: int64\n",
      "\n",
      "device_id属性的不同取值和出现的次数\n",
      "a99f214a    85145\n",
      "936e92fb       41\n",
      "6784a088       41\n",
      "93cb7907       39\n",
      "bcf28145       38\n",
      "44ab35bd       35\n",
      "9c2c3afd       31\n",
      "afeffc18       26\n",
      "bba1945c       22\n",
      "dcdc860a       21\n",
      "17994726       16\n",
      "57beb52b       16\n",
      "03942b86       14\n",
      "a5cd53ed       14\n",
      "1ade5b7d       14\n",
      "4478e16a       13\n",
      "03559b29       13\n",
      "20d10866       13\n",
      "6280ce87       12\n",
      "68dda5f6       12\n",
      "533b83bd       12\n",
      "d330699f       12\n",
      "e729151d       12\n",
      "d8eb3f98       12\n",
      "5697b7dc       11\n",
      "f9a3d527       11\n",
      "3b7ff62d       11\n",
      "978e0fa4       11\n",
      "7bf17714       10\n",
      "13a8551f       10\n",
      "            ...  \n",
      "2db29b7f        1\n",
      "68dbfdef        1\n",
      "86997780        1\n",
      "b005afba        1\n",
      "c9458931        1\n",
      "f10b4b4e        1\n",
      "6c49db90        1\n",
      "2e6f1902        1\n",
      "985643f4        1\n",
      "811ccf6f        1\n",
      "c4f4569a        1\n",
      "321669de        1\n",
      "9eb4bcf1        1\n",
      "3d54d4a3        1\n",
      "96305cc6        1\n",
      "4e027352        1\n",
      "c68a0bc2        1\n",
      "a5ad6ee0        1\n",
      "466bc98e        1\n",
      "d3a6e39e        1\n",
      "3032a487        1\n",
      "a39fcdd6        1\n",
      "4242f2be        1\n",
      "078d8f54        1\n",
      "090cc8d1        1\n",
      "a6006097        1\n",
      "76c63c70        1\n",
      "5f060b6f        1\n",
      "e68cad88        1\n",
      "3e454042        1\n",
      "Name: device_id, Length: 11500, dtype: int64\n",
      "\n",
      "device_ip属性的不同取值和出现的次数\n",
      "6b9769f2    553\n",
      "431b3174    337\n",
      "ee0389c1    266\n",
      "1cf29716    204\n",
      "57cd4006    199\n",
      "0489ce3f    199\n",
      "ddd2926e    195\n",
      "75bb1b58    184\n",
      "488a9a3e    184\n",
      "c6563308    176\n",
      "8a014cbb    175\n",
      "ceffea69    175\n",
      "9b1fe278    163\n",
      "116e4cf3    163\n",
      "a8536f3a    162\n",
      "07875ea4    143\n",
      "9aa18474    129\n",
      "7818404b    101\n",
      "884297d2     95\n",
      "bca8f26d     94\n",
      "ff1c4f79     87\n",
      "ac77b71a     85\n",
      "e54c1344     84\n",
      "95b2935e     79\n",
      "b0070d9a     71\n",
      "7ed30f6c     69\n",
      "693bff3e     67\n",
      "59ac940d     59\n",
      "aa42be81     57\n",
      "51b9d103     56\n",
      "           ... \n",
      "60bb8578      1\n",
      "0fdd0656      1\n",
      "80e725ac      1\n",
      "22183f3a      1\n",
      "dcc309ac      1\n",
      "77259d82      1\n",
      "14d84aff      1\n",
      "b273298d      1\n",
      "5f1038fc      1\n",
      "8776ddf2      1\n",
      "476b154a      1\n",
      "2ce63c97      1\n",
      "85743b19      1\n",
      "602b300f      1\n",
      "340727e7      1\n",
      "5cbefc7b      1\n",
      "ff680a59      1\n",
      "1d3b5bd7      1\n",
      "a81db72c      1\n",
      "c44e647f      1\n",
      "8f033154      1\n",
      "2aed5f82      1\n",
      "55b1380b      1\n",
      "736930ba      1\n",
      "def86095      1\n",
      "11789416      1\n",
      "ff7e2406      1\n",
      "40fd8f8c      1\n",
      "311109b4      1\n",
      "e47a8382      1\n",
      "Name: device_ip, Length: 64904, dtype: int64\n",
      "\n",
      "device_model属性的不同取值和出现的次数\n",
      "8a4875bd    6187\n",
      "d787e91b    4000\n",
      "1f0bc64f    3929\n",
      "76dc4769    2066\n",
      "be6db1d7    1908\n",
      "4ea23a13    1797\n",
      "36b67a2a    1702\n",
      "a0f5f879    1526\n",
      "ecb851b2    1445\n",
      "7abbbd5c    1385\n",
      "1ccc7835    1385\n",
      "711ee120    1239\n",
      "c6263d8a    1172\n",
      "5096d134    1143\n",
      "99e427c9    1119\n",
      "3bd9e8e7    1098\n",
      "aad45b01    1011\n",
      "5db079b5     944\n",
      "a5bce124     919\n",
      "d4897fef     908\n",
      "0eb711ec     907\n",
      "36d749e5     877\n",
      "779d90c2     865\n",
      "be74e6fe     860\n",
      "ef726eae     802\n",
      "9e3836ff     787\n",
      "2ea4f8ba     769\n",
      "fce66524     748\n",
      "158e4944     685\n",
      "2203a096     617\n",
      "            ... \n",
      "fd9bed57       1\n",
      "87579dda       1\n",
      "e52c47ad       1\n",
      "4205bc73       1\n",
      "4156c5bc       1\n",
      "4f5081ea       1\n",
      "c57e9a04       1\n",
      "e5393fc6       1\n",
      "17d9100f       1\n",
      "f223742e       1\n",
      "fcfcc0f9       1\n",
      "0a2933a8       1\n",
      "66553d7d       1\n",
      "b357fe47       1\n",
      "b4196eca       1\n",
      "a8bd6201       1\n",
      "cd5b0b7a       1\n",
      "182ecb3f       1\n",
      "19dfeeb9       1\n",
      "eec2bad8       1\n",
      "f16efb5e       1\n",
      "c90d3832       1\n",
      "7455fd39       1\n",
      "bc43f83c       1\n",
      "d29924a0       1\n",
      "673729ee       1\n",
      "cc51a2e1       1\n",
      "e823fe9a       1\n",
      "3e44daf1       1\n",
      "a32f03a3       1\n",
      "Name: device_model, Length: 2737, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#查看所有类型的值分布\n",
    "categorical_features = test.select_dtypes(include=[\"object\"]).columns\n",
    "\n",
    "\n",
    "for col in categorical_features:\n",
    "    print( '\\n%s属性的不同取值和出现的次数'%col)\n",
    "    print (test[col].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 100000 entries, 186848 to 537696\n",
      "Data columns (total 24 columns):\n",
      "id                  100000 non-null float64\n",
      "click               100000 non-null int64\n",
      "hour                100000 non-null int64\n",
      "C1                  100000 non-null int64\n",
      "banner_pos          100000 non-null int64\n",
      "site_id             100000 non-null object\n",
      "site_domain         100000 non-null object\n",
      "site_category       100000 non-null object\n",
      "app_id              100000 non-null object\n",
      "app_domain          100000 non-null object\n",
      "app_category        100000 non-null object\n",
      "device_id           100000 non-null object\n",
      "device_ip           100000 non-null object\n",
      "device_model        100000 non-null object\n",
      "device_type         100000 non-null int64\n",
      "device_conn_type    100000 non-null int64\n",
      "C14                 100000 non-null int64\n",
      "C15                 100000 non-null int64\n",
      "C16                 100000 non-null int64\n",
      "C17                 100000 non-null int64\n",
      "C18                 100000 non-null int64\n",
      "C19                 100000 non-null int64\n",
      "C20                 100000 non-null int64\n",
      "C21                 100000 non-null int64\n",
      "dtypes: float64(1), int64(14), object(9)\n",
      "memory usage: 19.1+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 100000 entries, 186848 to 537696\n",
      "Data columns (total 24 columns):\n",
      "id                  100000 non-null float64\n",
      "click               100000 non-null int64\n",
      "hour                100000 non-null int64\n",
      "C1                  100000 non-null int64\n",
      "banner_pos          100000 non-null int64\n",
      "site_id             100000 non-null object\n",
      "site_domain         100000 non-null object\n",
      "site_category       100000 non-null object\n",
      "app_id              100000 non-null object\n",
      "app_domain          100000 non-null object\n",
      "app_category        100000 non-null object\n",
      "device_id           100000 non-null object\n",
      "device_ip           100000 non-null object\n",
      "device_model        100000 non-null object\n",
      "device_type         100000 non-null int64\n",
      "device_conn_type    100000 non-null int64\n",
      "C14                 100000 non-null int64\n",
      "C15                 100000 non-null int64\n",
      "C16                 100000 non-null int64\n",
      "C17                 100000 non-null int64\n",
      "C18                 100000 non-null int64\n",
      "C19                 100000 non-null int64\n",
      "C20                 100000 non-null int64\n",
      "C21                 100000 non-null int64\n",
      "dtypes: float64(1), int64(14), object(9)\n",
      "memory usage: 19.1+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 100000 entries, 536962 to 840172\n",
      "Data columns (total 23 columns):\n",
      "id                  100000 non-null float64\n",
      "hour                100000 non-null int64\n",
      "C1                  100000 non-null int64\n",
      "banner_pos          100000 non-null int64\n",
      "site_id             100000 non-null object\n",
      "site_domain         100000 non-null object\n",
      "site_category       100000 non-null object\n",
      "app_id              100000 non-null object\n",
      "app_domain          100000 non-null object\n",
      "app_category        100000 non-null object\n",
      "device_id           100000 non-null object\n",
      "device_ip           100000 non-null object\n",
      "device_model        100000 non-null object\n",
      "device_type         100000 non-null int64\n",
      "device_conn_type    100000 non-null int64\n",
      "C14                 100000 non-null int64\n",
      "C15                 100000 non-null int64\n",
      "C16                 100000 non-null int64\n",
      "C17                 100000 non-null int64\n",
      "C18                 100000 non-null int64\n",
      "C19                 100000 non-null int64\n",
      "C20                 100000 non-null int64\n",
      "C21                 100000 non-null int64\n",
      "dtypes: float64(1), int64(13), object(9)\n",
      "memory usage: 18.3+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将时间按小时来区分，分为24小时"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#对时间进行预处理\n",
    "def procedess_hour(df):\n",
    "    df['hour']=df['hour']%100\n",
    "    df['hour_0'] = df['hour'].apply(lambda x : 1 if x==0 else 0)\n",
    "    df['hour_1'] = df['hour'].apply(lambda x : 1 if x==1 else 0)\n",
    "    df['hour_2'] = df['hour'].apply(lambda x : 1 if x==2 else 0)\n",
    "    df['hour_3'] = df['hour'].apply(lambda x : 1 if x==3 else 0)\n",
    "    df['hour_4'] = df['hour'].apply(lambda x : 1 if x==4 else 0)\n",
    "    df['hour_5'] = df['hour'].apply(lambda x : 1 if x==5 else 0)\n",
    "    df['hour_6'] = df['hour'].apply(lambda x : 1 if x==6 else 0)\n",
    "    df['hour_7'] = df['hour'].apply(lambda x : 1 if x==7 else 0)\n",
    "    df['hour_8'] = df['hour'].apply(lambda x : 1 if x==8 else 0)\n",
    "    df['hour_9'] = df['hour'].apply(lambda x : 1 if x==9 else 0)\n",
    "    df['hour_10'] = df['hour'].apply(lambda x : 1 if x==10 else 0)\n",
    "    df['hour_11'] = df['hour'].apply(lambda x : 1 if x==11 else 0)\n",
    "    df['hour_12'] = df['hour'].apply(lambda x : 1 if x==12 else 0)\n",
    "    df['hour_13'] = df['hour'].apply(lambda x : 1 if x==13 else 0)\n",
    "    df['hour_14'] = df['hour'].apply(lambda x : 1 if x==14 else 0)\n",
    "    df['hour_15'] = df['hour'].apply(lambda x : 1 if x==15 else 0)\n",
    "    df['hour_16'] = df['hour'].apply(lambda x : 1 if x==16 else 0)\n",
    "    df['hour_17'] = df['hour'].apply(lambda x : 1 if x==17 else 0)\n",
    "    df['hour_18'] = df['hour'].apply(lambda x : 1 if x==18 else 0)\n",
    "    df['hour_19'] = df['hour'].apply(lambda x : 1 if x==19 else 0)\n",
    "    df['hour_20'] = df['hour'].apply(lambda x : 1 if x==20 else 0)\n",
    "    df['hour_21'] = df['hour'].apply(lambda x : 1 if x==21 else 0)\n",
    "    df['hour_22'] = df['hour'].apply(lambda x : 1 if x==22 else 0)\n",
    "    df['hour_23'] = df['hour'].apply(lambda x : 1 if x==23 else 0)\n",
    "    \n",
    "    df.drop(['hour'], axis=1,inplace = True)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对站点ID进行处理\n",
    "将state_id 分为几个等级 top 1%， 2%， 5， 10， 15， 20， 25， 30， 50，"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_site_id (df):\n",
    "    site_count = df['site_id'].value_counts()\n",
    "\n",
    "    df['top_10_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_siter'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_ssite'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_site'] = df['site_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['site_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对站点领域进行处理\n",
    "将state_domain 分为几个等级 top 1%， 2%， 5， 10， 15， 20， 25， 30， 50，"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_site_domain (df):\n",
    "    site_count = df['site_domain'].value_counts()\n",
    "\n",
    "    df['top_10_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_site_domain'] = df['site_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['site_domain'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对站点类型进行处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_site_category (df):\n",
    "    site_count = df['site_category'].value_counts()\n",
    "\n",
    "    df['top_10_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_site_category'] = df['site_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['site_category'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对APP_id 进行预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_app_id (df):\n",
    "    site_count = df['app_id'].value_counts()\n",
    "\n",
    "    df['top_10_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_app_id'] = df['app_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['app_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对app_domain进行处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_app_domain (df):\n",
    "    site_count = df['app_domain'].value_counts()\n",
    "\n",
    "    df['top_10_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_app_domain'] = df['app_domain'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['app_domain'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对app_category 进行预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_app_category (df):\n",
    "    site_count = df['app_category'].value_counts()\n",
    "\n",
    "    df['top_10_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_app_category'] = df['app_category'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['app_category'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对device_id 进行预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_device_id (df):\n",
    "    site_count = df['device_id'].value_counts()\n",
    "\n",
    "    df['top_10_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_device_id'] = df['device_id'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['device_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对device_ip 进行预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_device_ip (df):\n",
    "    site_count = df['device_ip'].value_counts()\n",
    "\n",
    "    df['top_10_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_device_ip'] = df['device_ip'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['device_ip'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对device_model 进行预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_device_model (df):\n",
    "    site_count = df['device_model'].value_counts()\n",
    "\n",
    "    df['top_10_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 90)] else 0)\n",
    "    \n",
    "    df['top_25_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 75)] else 0)\n",
    "    \n",
    "    df['top_5_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 95)] else 0)\n",
    "    \n",
    "    df['top_50_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 50)] else 0)\n",
    "    \n",
    "    df['top_1_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 99)] else 0)\n",
    "    \n",
    "    df['top_2_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 98)] else 0)\n",
    "    \n",
    "    df['top_15_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 85)] else 0)\n",
    "    \n",
    "    df['top_20_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 80)] else 0)\n",
    "    \n",
    "    df['top_30_device_model'] = df['device_model'].apply(lambda x: 1 if x in site_count.index.values[\n",
    "        site_count.values >= np.percentile(site_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['device_model'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对训练数据和测试数据进行预处理，并进行one_hot编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "procedess_hour(train)\n",
    "procdess_site_id(train)\n",
    "procdess_site_domain(train)\n",
    "procdess_site_category(train)\n",
    "procdess_app_id(train)\n",
    "procdess_app_domain(train)\n",
    "procdess_app_category(train)\n",
    "procdess_device_id(train)\n",
    "procdess_device_ip(train)\n",
    "procdess_device_model(train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>click</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>device_type</th>\n",
       "      <th>device_conn_type</th>\n",
       "      <th>C14</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>C17</th>\n",
       "      <th>...</th>\n",
       "      <th>top_30_device_ip</th>\n",
       "      <th>top_10_device_model</th>\n",
       "      <th>top_25_device_model</th>\n",
       "      <th>top_5_device_model</th>\n",
       "      <th>top_50_device_model</th>\n",
       "      <th>top_1_device_model</th>\n",
       "      <th>top_2_device_model</th>\n",
       "      <th>top_15_device_model</th>\n",
       "      <th>top_20_device_model</th>\n",
       "      <th>top_30_device_model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>186848</th>\n",
       "      <td>1.819859e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>20633</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2374</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815809</th>\n",
       "      <td>3.357563e+18</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15706</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51654</th>\n",
       "      <td>1.719468e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21611</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2480</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979514</th>\n",
       "      <td>1.349057e+19</td>\n",
       "      <td>0</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19251</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2201</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58702</th>\n",
       "      <td>1.817488e+19</td>\n",
       "      <td>1</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15704</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>1722</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 119 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id  click    C1  banner_pos  device_type  device_conn_type  \\\n",
       "186848  1.819859e+19      0  1005           0            1                 0   \n",
       "815809  3.357563e+18      0  1005           0            1                 0   \n",
       "51654   1.719468e+19      0  1005           0            1                 0   \n",
       "979514  1.349057e+19      0  1005           0            1                 0   \n",
       "58702   1.817488e+19      1  1005           0            1                 0   \n",
       "\n",
       "          C14  C15  C16   C17         ...           top_30_device_ip  \\\n",
       "186848  20633  320   50  2374         ...                          1   \n",
       "815809  15706  320   50  1722         ...                          1   \n",
       "51654   21611  320   50  2480         ...                          1   \n",
       "979514  19251  320   50  2201         ...                          1   \n",
       "58702   15704  320   50  1722         ...                          1   \n",
       "\n",
       "        top_10_device_model  top_25_device_model  top_5_device_model  \\\n",
       "186848                    0                    0                   0   \n",
       "815809                    1                    1                   1   \n",
       "51654                     1                    1                   1   \n",
       "979514                    1                    1                   1   \n",
       "58702                     1                    1                   1   \n",
       "\n",
       "        top_50_device_model  top_1_device_model  top_2_device_model  \\\n",
       "186848                    1                   0                   0   \n",
       "815809                    1                   1                   1   \n",
       "51654                     1                   1                   1   \n",
       "979514                    1                   0                   0   \n",
       "58702                     1                   1                   1   \n",
       "\n",
       "        top_15_device_model  top_20_device_model  top_30_device_model  \n",
       "186848                    0                    0                    0  \n",
       "815809                    1                    1                    1  \n",
       "51654                     1                    1                    1  \n",
       "979514                    1                    1                    1  \n",
       "58702                     1                    1                    1  \n",
       "\n",
       "[5 rows x 119 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 100000 entries, 186848 to 537696\n",
      "Columns: 119 entries, id to top_30_device_model\n",
      "dtypes: float64(1), int64(118)\n",
      "memory usage: 91.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 保存特征工程后的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.to_csv('train_FE.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 对测试数据进行特征工程处理\n",
    "procedess_hour(test)\n",
    "procdess_site_id(test)\n",
    "procdess_site_domain(test)\n",
    "procdess_site_category(test)\n",
    "procdess_app_id(test)\n",
    "procdess_app_domain(test)\n",
    "procdess_app_category(test)\n",
    "procdess_device_id(test)\n",
    "procdess_device_ip(test)\n",
    "procdess_device_model(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>C1</th>\n",
       "      <th>banner_pos</th>\n",
       "      <th>device_type</th>\n",
       "      <th>device_conn_type</th>\n",
       "      <th>C14</th>\n",
       "      <th>C15</th>\n",
       "      <th>C16</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>...</th>\n",
       "      <th>top_30_device_ip</th>\n",
       "      <th>top_10_device_model</th>\n",
       "      <th>top_25_device_model</th>\n",
       "      <th>top_5_device_model</th>\n",
       "      <th>top_50_device_model</th>\n",
       "      <th>top_1_device_model</th>\n",
       "      <th>top_2_device_model</th>\n",
       "      <th>top_15_device_model</th>\n",
       "      <th>top_20_device_model</th>\n",
       "      <th>top_30_device_model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>536962</th>\n",
       "      <td>1.060423e+18</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8330</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>761</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736381</th>\n",
       "      <td>1.368686e+17</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>24084</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2761</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330635</th>\n",
       "      <td>1.457154e+19</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>23137</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2664</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858388</th>\n",
       "      <td>4.625780e+18</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>22254</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2545</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>786323</th>\n",
       "      <td>1.677718e+19</td>\n",
       "      <td>1005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24132</td>\n",
       "      <td>320</td>\n",
       "      <td>50</td>\n",
       "      <td>2768</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 118 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id    C1  banner_pos  device_type  device_conn_type    C14  \\\n",
       "536962  1.060423e+18  1005           0            1                 0   8330   \n",
       "736381  1.368686e+17  1005           0            1                 2  24084   \n",
       "330635  1.457154e+19  1005           0            1                 0  23137   \n",
       "858388  4.625780e+18  1005           0            1                 0  22254   \n",
       "786323  1.677718e+19  1005           0            1                 0  24132   \n",
       "\n",
       "        C15  C16   C17  C18         ...           top_30_device_ip  \\\n",
       "536962  320   50   761    3         ...                          1   \n",
       "736381  320   50  2761    2         ...                          1   \n",
       "330635  320   50  2664    0         ...                          1   \n",
       "858388  320   50  2545    0         ...                          1   \n",
       "786323  320   50  2768    1         ...                          1   \n",
       "\n",
       "        top_10_device_model  top_25_device_model  top_5_device_model  \\\n",
       "536962                    1                    1                   1   \n",
       "736381                    1                    1                   1   \n",
       "330635                    1                    1                   1   \n",
       "858388                    1                    1                   1   \n",
       "786323                    1                    1                   1   \n",
       "\n",
       "        top_50_device_model  top_1_device_model  top_2_device_model  \\\n",
       "536962                    1                   1                   1   \n",
       "736381                    1                   0                   1   \n",
       "330635                    1                   1                   1   \n",
       "858388                    1                   1                   1   \n",
       "786323                    1                   0                   1   \n",
       "\n",
       "        top_15_device_model  top_20_device_model  top_30_device_model  \n",
       "536962                    1                    1                    1  \n",
       "736381                    1                    1                    1  \n",
       "330635                    1                    1                    1  \n",
       "858388                    1                    1                    1  \n",
       "786323                    1                    1                    1  \n",
       "\n",
       "[5 rows x 118 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.to_csv('test_FE.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1005    93640\n",
       "1002     3158\n",
       "1010     1764\n",
       "1012     1180\n",
       "1008      153\n",
       "1007       94\n",
       "1001       11\n",
       "Name: C1, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = train['C1']\n",
    "data.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1005    93690\n",
       "1002     4077\n",
       "1010     1621\n",
       "1012      577\n",
       "1007       20\n",
       "1001       15\n",
       "Name: C1, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['C1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
